import re
import json
import hashlib
import base64
import urllib.parse
from datetime import datetime
from app.utils.logger import logger

# SQL注入检测模式
sql_injection_patterns = [
    r"'\s*OR\s*'1'='1",
    r"'\s*OR\s*1=1",
    r"\s*OR\s*1=1\s*--",
    r"'\s*UNION\s*SELECT",
    r"\s*UNION\s*SELECT\s*1",
    r"'\s*AND\s*'1'='1",
    r"'\s*DROP\s*TABLE",
    r"exec\(sp_executesql",
    r"1=1\s+OR\s+'a'='a",
    r"SELECT\s+\*\s+FROM\s+",
    r"INSERT\s+INTO\s+",
    r"UPDATE\s+.+\s+SET\s+"
]

# XSS检测模式
xss_patterns = [
    '<script>.*?</script>',
    '<img\\s+src=["\\\']?javascript:.*?>',
    '<svg\\s+onload=["\\\']?.*?>',
    '<iframe\\s+src=["\\\']?.*?>',
    '<body\\s+onload=["\\\']?.*?>',
    'alert\\(.*?\\)',
    'confirm\\(.*?\\)',
    'prompt\\(.*?\\)',
    'document\\.cookie',
    'document\\.location',
    'eval\\(.*?\\)',
    'String\\.fromCharCode\\('
]

# 常见信息泄露文件路径
common_info_files = [
    '.git/config',
    '.env',
    '.bashrc',
    '.bash_profile',
    'config/db.php',
    'wp-config.php',
    'web.config',
    'robots.txt',
    'sitemap.xml',
    '.DS_Store',
    '.svn/entries',
    '.hg/branch',
    'composer.json',
    'package.json',
    'requirements.txt'
]

def detect_sql_injection(response_text):
    """
    检测响应中是否包含SQL注入的迹象 - 增强版
    :param response_text: HTTP响应文本
    :return: 匹配的模式列表（包含详细信息）
    """
    if not response_text:
        return []
    
    matches = []
    response_lower = response_text.lower()
    
    # 1. 检查基本注入模式
    # 扁平化注入模式字典
    all_patterns = []
    if isinstance(sql_injection_patterns, dict):
        for category, patterns in sql_injection_patterns.items():
            if isinstance(patterns, dict):
                for subcategory, subpatterns in patterns.items():
                    all_patterns.extend([(p, f"{category}_{subcategory}") for p in subpatterns])
            else:
                all_patterns.extend([(p, category) for p in patterns])
    else:
        # 兼容旧格式
        all_patterns = [(p, "basic") for p in sql_injection_patterns]
    
    for pattern, category in all_patterns:
        if re.search(pattern, response_text, re.IGNORECASE):
            matches.append(f"[{category}] {pattern}")
    
    # 2. 检查数据库特定错误消息
    db_errors = {
        'mysql': [
            "mysql error", "mysql_fetch", "mysql_num_rows",
            "error in your sql syntax", "syntax error near",
            "you have an error in your sql syntax", "mysql server",
            "unclosed quotation mark", "column not found",
            "mysql_query", "undefined index", "division by zero",
            "duplicate entry", "incorrect integer value",
            "subquery returns more than 1 row"
        ],
        'postgresql': [
            "postgresql error", "pg_query", "pg_exec",
            "syntax error at or near", "unterminated quoted string",
            "relation .* does not exist", "column .* does not exist",
            "operator does not exist", "invalid input syntax",
            "function .* does not exist", "cannot cast type"
        ],
        'mssql': [
            "microsoft sql", "t-sql", "transact-sql",
            "incorrect syntax near", "unclosed quotation mark",
            "invalid column name", "invalid object name",
            "string or binary data would be truncated",
            "permission denied", "unexpected end of file"
        ],
        'oracle': [
            "ora-\d+", "pl/sql", "oracle error",
            "invalid identifier", "missing comma",
            "string literal too long", "no data found",
            "too many values", "exact fetch returns more than requested number of rows"
        ],
        'sqlite': [
            'sqlite error', 'sqlite3', 'unrecognized token',
            'near ".*": syntax error', 'no such table',
            'no such column', 'unclosed quote'
        ]
    }
    
    for db_type, errors in db_errors.items():
        for error in errors:
            if re.search(error.lower(), response_lower):
                matches.append(f"[error_{db_type}] SQL error: {error}")
    
    # 3. 检查通用SQL错误模式
    general_errors = [
        "sql syntax", "database error", "query error",
        "syntax error", "unclosed quotation", "column not found",
        "table not found", "invalid query", "exception occurred",
        "database exception", "query failed", "invalid sql",
        "unterminated string", "unexpected end", "invalid syntax"
    ]
    
    for error in general_errors:
        if error in response_lower:
            matches.append(f"[error_general] SQL error: {error}")
    
    # 4. 检查数据库信息泄露
    db_info_patterns = [
        r"mysql\s+version", r"postgresql\s+version", r"oracle\s+version",
        r"sql server\s+version", r"database name", r"server name",
        r"connection string", r"jdbc:mysql", r"jdbc:postgresql",
        r"jdbc:oracle", r"jdbc:sqlserver", r"sqlite://",
        r"information_schema", r"sys.tables", r"pg_tables",
        r"user\(\)", r"version\(\)", r"database\(\)"
    ]
    
    for pattern in db_info_patterns:
        if re.search(pattern, response_lower):
            matches.append(f"[info_leak] Database info exposed: {pattern}")
    
    # 5. 检查盲注特征（响应差异等）
    blind_indicators = [
        r"(you have an error|syntax error).*?\'", r"unterminated string constant",
        r"unexpected end of sql command", r"missing right parenthesis",
        r"data type mismatch", r"operand should contain",
        r"invalid operator", r"out of range value",
        r"division by zero", r"undefined index",
        r"trying to get property of non-object"
    ]
    
    for pattern in blind_indicators:
        if re.search(pattern, response_lower):
            matches.append(f"[blind] Blind SQL injection indicator: {pattern}")
    
    # 去重并限制返回的结果数量
    unique_matches = list(set(matches))[:30]
    
    # 按类型排序，将错误信息放在前面
    unique_matches.sort(key=lambda x: "error" in x.lower(), reverse=True)
    
    return unique_matches

def detect_xss(response_text, payload):
    """
    检测响应中是否包含XSS漏洞的迹象 - 增强版
    :param response_text: HTTP响应文本
    :param payload: 发送的XSS测试载荷
    :return: 是否检测到XSS
    """
    if not response_text or not payload:
        return False
    
    # 1. 基础检测 - 原始payload是否在响应中出现
    if payload in response_text:
        return True
    
    # 2. 检测部分payload匹配（处理payload被截断的情况）
    # 提取payload中的关键字段
    payload_keywords = [
        'script', 'alert', 'onerror', 'onload', 'onclick',
        'javascript', 'iframe', 'svg', 'eval', 'document.cookie',
        'confirm', 'prompt', 'location', 'xss', 'html', 'body'
    ]
    
    payload_lower = payload.lower()
    response_lower = response_text.lower()
    matched_keywords = []
    
    for keyword in payload_keywords:
        if keyword in payload_lower and keyword in response_lower:
            matched_keywords.append(keyword)
    
    # 如果匹配到多个关键字，增加检测敏感度
    if len(matched_keywords) > 2:
        # 检查这些关键字是否在相近的位置出现
        keyword_positions = []
        for keyword in matched_keywords[:3]:  # 只检查前3个最关键的
            start = 0
            while True:
                pos = response_lower.find(keyword, start)
                if pos == -1:
                    break
                keyword_positions.append(pos)
                start = pos + 1
        
        # 如果有多个关键字在100个字符范围内出现，认为可能存在XSS
        if len(keyword_positions) >= 2:
            keyword_positions.sort()
            if keyword_positions[-1] - keyword_positions[0] < 100:
                return True
    
    # 3. 检测编码绕过
    # URL编码检测
    encoded_payload = urllib.parse.quote(payload)
    if encoded_payload in response_text:
        return True
    
    # 双URL编码检测
    double_encoded = urllib.parse.quote(encoded_payload)
    if double_encoded in response_text:
        return True
    
    # 4. 检测HTML实体编码绕过
    html_entities = {
        '<': ['&lt;', '&#60;', '&#x3c;', '&#x003c;', '&#060;'],
        '>': ['&gt;', '&#62;', '&#x3e;', '&#x003e;', '&#062;'],
        '"': ['&quot;', '&#34;', '&#x22;', '&#x0022;', '&#034;'],
        "'": ['&#39;', '&#x27;', '&#x0027;', '&#039;'],
        '(': ['&#40;', '&#x28;', '&#x0028;', '&#040;'],
        ')': ['&#41;', '&#x29;', '&#x0029;', '&#041;']
    }
    
    # 检查是否存在部分编码的payload
    for char, entities in html_entities.items():
        if char in payload:
            for entity in entities:
                if entity in response_text:
                    # 检查上下文，看是否可能是有效的XSS
                    context_pattern = r'.*?' + re.escape(entity) + r'.*?'
                    match = re.search(context_pattern, response_text, re.IGNORECASE)
                    if match:
                        context = match.group(0).lower()
                        # 检查是否在HTML标签或属性中
                        if any(tag in context for tag in ['<script', '</script', 'onerror=', 'onload=', 'onclick=', 'javascript:', 'expression(']):
                            return True
    
    # 5. 检查常见XSS执行特征
    # 扁平化XSS模式字典
    all_xss_patterns = []
    if isinstance(xss_patterns, dict):
        for category, patterns in xss_patterns.items():
            all_xss_patterns.extend(patterns)
    else:
        # 兼容旧格式
        all_xss_patterns = xss_patterns
    
    for pattern in all_xss_patterns:
        if re.search(pattern, response_text, re.IGNORECASE):
            # 进一步验证上下文，减少误报
            context_pattern = r'.*?' + pattern + r'.*?'
            match = re.search(context_pattern, response_text, re.IGNORECASE)
            if match:
                context = match.group(0)
                # 检查是否在注释或CDATA中
                if '<!--' not in context and '//<![CDATA[' not in context:
                    # 如果payload的某些部分也在上下文中，提高置信度
                    payload_parts = re.split(r'[<>"\'\\\s]', payload)
                    significant_parts = [p for p in payload_parts if len(p) > 2]
                    
                    if not significant_parts or any(part.lower() in context.lower() for part in significant_parts):
                        return True
    
    # 6. 高级检测：检查特定JavaScript函数调用
    js_functions = ['alert', 'confirm', 'prompt', 'eval', 'document.write', 
                   'document.cookie', 'localStorage', 'sessionStorage',
                   'window.location', 'document.location', 'XMLHttpRequest',
                   'fetch', 'setTimeout', 'setInterval', 'Function']
    
    for func in js_functions:
        # 检查函数调用模式
        func_pattern = r'\b' + re.escape(func) + r'\s*\('  # 例如 alert(
        if re.search(func_pattern, response_lower):
            # 检查是否与payload相关
            payload_parts = re.split(r'[<>"\'\\\s]', payload)
            significant_parts = [p for p in payload_parts if len(p) > 2]
            
            if any(part.lower() in response_lower for part in significant_parts):
                return True
    
    # 7. 检测DOM型XSS特征
    dom_xss_patterns = [
        r'document\.write\s*\(', r'innerhtml\s*=', r'outerhtml\s*=',
        r'document\.createelement\s*\(\s*["\']script["\']\s*\)',
        r'eval\s*\(', r'function\s*\(\)\s*{[^}]*document\.cookie',
        r'document\.location\s*=', r'window\.open\s*\(',
        r'appendchild\s*\(', r'insertbefore\s*\(', r'replacechild\s*\(',
        r'createtextnode\s*\(', r'document\.getelementbyid\s*\(.*\)\.value',
        r'document\.getelementsbyname\s*\(.*\)\[\d+\]\.value'
    ]
    
    for pattern in dom_xss_patterns:
        if re.search(pattern, response_lower):
            # 再次检查payload部分是否存在
            payload_parts = re.split(r'[<>"\'\\\s]', payload)
            significant_parts = [p for p in payload_parts if len(p) > 2]
            
            if any(part.lower() in response_lower for part in significant_parts):
                return True
    
    return False

def check_sensitive_files(base_url):
    """
    检查网站是否存在敏感文件 - 增强版
    :param base_url: 基础URL
    :return: 发现的敏感文件列表
    """
    import re
    import time
    import requests
    from app.utils.http_utils import make_request
    from app.utils.logger import logger
    
    # 分类敏感文件路径
    sensitive_files = {
        'common': [
            'robots.txt',
            'sitemap.xml',
            '.htaccess',
            '.htpasswd',
            '.git/HEAD',
            '.git/config',
            '.git/description',
            '.git/info/refs',
            '.svn/entries',
            '.svn/wc.db',
            '.hg/last-message.txt',
            '.DS_Store',
            'Thumbs.db',
            'web.config',
            'php.ini',
            'nginx.conf',
            'apache2.conf'
        ],
        'backup': [
            'backup/',
            'backups/',
            'backup.zip',
            'backup.tar.gz',
            'backup.sql',
            'backup.dump',
            'database.sql',
            'db_backup/',
            'mysql.dump',
            'old_backup/',
            'backup_',
            '_backup/',
            '.backup/'
        ],
        'configuration': [
            'config.php',
            'wp-config.php',
            'settings.php',
            'config.ini',
            'config.json',
            'config.yml',
            'application.properties',
            'application.yml',
            'settings.json',
            'local_settings.py',
            'secrets.json',
            'database.yml',
            'db.php',
            'db_connect.php',
            'database.php'
        ],
        'information_disclosure': [
            'phpinfo.php',
            'info.php',
            'php_info.php',
            'info.php5',
            'info.phtml',
            'test.php',
            'test.html',
            'test.txt',
            'debug.php',
            'status.php',
            'stats.php',
            'about.php',
            'version.php',
            'env.php',
            'environment.php',
            'var_dump.php'
        ],
        'log_files': [
            'error.log',
            'access.log',
            'app.log',
            'application.log',
            'server.log',
            'mysql.log',
            'php.log',
            'apache.log',
            'nginx.log',
            'system.log',
            'debug.log',
            'exception.log',
            'logs/',
            'log/'
        ],
        'admin': [
            'admin/',
            'admin/login.php',
            'admin/index.php',
            'admin/admin.php',
            'admin/login.html',
            'administration/',
            'adminpanel/',
            'admincp/',
            'admin_area/',
            'admin_area/login.php',
            'admin_area/index.php',
            'cms/login.php',
            'cms/index.php',
            'backend/',
            'backend/login.php',
            'wp-admin/',
            'wp-login.php',
            'drupal/login.php',
            'joomla/administrator/'
        ],
        'development': [
            '.vscode/',
            '.idea/',
            'node_modules/',
            'bower_components/',
            'composer.json',
            'package.json',
            'package-lock.json',
            'yarn.lock',
            'requirements.txt',
            'Gemfile',
            'Pipfile',
            '.env',
            '.env.example',
            '.env.local',
            '.env.development',
            '.env.production',
            '.travis.yml',
            'Jenkinsfile',
            '.gitlab-ci.yml',
            'docker-compose.yml',
            'Dockerfile'
        ],
        'upload': [
            'uploads/',
            'upload/',
            'images/',
            'files/',
            'documents/',
            'media/',
            'downloads/',
            'tmp/',
            'temp/',
            'cache/'
        ]
    }
    
    found_files = []
    # 更详细的请求头以模拟真实浏览器
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5',
        'Accept-Encoding': 'gzip, deflate',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1',
        'Cache-Control': 'max-age=0'
    }
    
    # 扁平化所有敏感文件路径
    all_files = []
    for category, files in sensitive_files.items():
        for file_path in files:
            all_files.append((file_path, category))
    
    # 添加常见目录遍历尝试
    directory_traversal_payloads = [
        '../etc/passwd',
        '../../etc/passwd',
        '../../../etc/passwd',
        '..\\..\\..\\windows\\win.ini',
        '..%2f..%2f..%2fetc%2fpasswd',
        '%2e%2e%2fetc%2fpasswd',
        '%2e%2e%5c%2e%2e%5c%2e%2e%5cwindows%5cwin.ini'
    ]
    
    for payload in directory_traversal_payloads:
        all_files.append((payload, 'directory_traversal'))
    
    # 实现重试机制和并发检查的准备
    max_retries = 2
    timeout = 5
    
    for file_path, category in all_files:
        url = f"{base_url.rstrip('/')}/{file_path.lstrip('/')}"
        retry_count = 0
        success = False
        
        while retry_count <= max_retries and not success:
            try:
                # 先尝试使用http_utils中的make_request函数
                response = make_request(url, method='HEAD', headers=headers, timeout=timeout, allow_redirects=True)
                
                # 如果失败，尝试直接使用requests库作为备选
                if not response:
                    response = requests.head(url, headers=headers, timeout=timeout, 
                                           allow_redirects=True, stream=True)
                
                # 检查状态码
                if 200 <= response.status_code < 400:
                    # 分析响应
                    content_type = response.headers.get('Content-Type', '').lower()
                    content_length = response.headers.get('Content-Length')
                    
                    # 对于可能包含敏感信息的文件，进行更深入的GET检查
                    # 特别是配置文件、日志文件和备份文件
                    if category in ['configuration', 'log_files', 'backup', 'information_disclosure']:
                        # 对这些高风险文件执行GET请求
                        get_response = make_request(url, method='GET', headers=headers, timeout=timeout,
                                                  allow_redirects=True, stream=True)
                        
                        if not get_response:
                            get_response = requests.get(url, headers=headers, timeout=timeout,
                                                      allow_redirects=True, stream=True)
                        
                        # 只读取前10KB以节省时间和带宽
                        get_response_content = get_response.raw.read(10240).decode('utf-8', errors='ignore')
                        
                        # 检查内容中是否包含敏感信息特征
                        is_sensitive = False
                        
                        # 针对不同文件类型的特征检查
                        if category == 'configuration':
                            # 配置文件敏感信息特征
                            sensitive_patterns = [
                                r'password\s*=\s*["\']',
                                r'pass\s*=\s*["\']',
                                r'pwd\s*=\s*["\']',
                                r'api[_-]?key\s*=\s*["\']',
                                r'secret\s*=\s*["\']',
                                r'private\s+key',
                                r'database[_-]?connection',
                                r'mysql[_-]?user',
                                r'host\s*=\s*["\']',
                                r'database\s*=\s*["\']',
                                r'connection\s*string',
                                r'jdbc:mysql://',
                                r'@mongodb:\\\\//',
                                r'postgresql:\\\\//'
                            ]
                            for pattern in sensitive_patterns:
                                if re.search(pattern, get_response_content, re.IGNORECASE):
                                    is_sensitive = True
                                    break
                        
                        elif category == 'log_files':
                            # 日志文件特征
                            log_patterns = [
                                r'error:',
                                r'exception:',
                                r'warning:',
                                r'stack trace:',
                                r'database error',
                                r'fatal:',
                                r'server error',
                                r'request:',
                                r'user-agent:',
                                r'remote address:',
                                r'http/1\\.\\d',
                                r'\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}',
                                r'\\[\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\]',
                                r'\\b(?:get|post|put|delete|head)\\s+\/[^\\s]+\\s+http\\/\\d\\.\\d\\b'
                            ]
                            for pattern in log_patterns:
                                if re.search(pattern, get_response_content, re.IGNORECASE):
                                    is_sensitive = True
                                    break
                        
                        elif category == 'backup':
                            # 备份文件特征
                            if any(ext in url.lower() for ext in ['.zip', '.tar', '.gz', '.sql', '.dump', '.bak']):
                                is_sensitive = True
                            # 检查内容是否为SQL格式或数据库转储格式
                            elif re.search(r'(?:create\\s+table|insert\\s+into|select\\s+\\*|drop\\s+table)', 
                                          get_response_content, re.IGNORECASE):
                                is_sensitive = True
                        
                        elif category == 'information_disclosure':
                            # 信息泄露特征
                            phpinfo_patterns = [
                                r'php\\s+version',
                                r'server\\s+software',
                                r'loaded\\s+extensions',
                                r'environment',
                                r'php\\s+info',
                                r'gd\\s+support',
                                r'mysql\\s+version',
                                r'postgresql\\s+version',
                                r'pdo\\s+drivers'
                            ]
                            for pattern in phpinfo_patterns:
                                if re.search(pattern, get_response_content, re.IGNORECASE):
                                    is_sensitive = True
                                    break
                        
                        # 针对Git和SVN目录检查
                        if file_path.startswith('.git/') or file_path.startswith('.svn/'):
                            # Git/SVN文件特征
                            git_patterns = [
                                r'ref:\\s+refs\\/',
                                r'object\\s+store',
                                r'packed\\s+refs',
                                r'branch\\s+heads',
                                r'tree\\s+[0-9a-f]{40}',
                                r'commit\\s+[0-9a-f]{40}'
                            ]
                            svn_patterns = [
                                r'<\?xml\\s+version',
                                r'<wc-entry',
                                r'<entry',
                                r'<name>.*?<\\/name>',
                                r'<url>.*?<\\/url>'
                            ]
                            
                            if file_path.startswith('.git/'):
                                for pattern in git_patterns:
                                    if re.search(pattern, get_response_content, re.IGNORECASE):
                                        is_sensitive = True
                                        break
                            elif file_path.startswith('.svn/'):
                                for pattern in svn_patterns:
                                    if re.search(pattern, get_response_content, re.IGNORECASE):
                                        is_sensitive = True
                                        break
                        
                        if is_sensitive:
                            found_files.append({
                                'url': url,
                                'status_code': get_response.status_code,
                                'content_type': content_type,
                                'content_length': content_length,
                                'category': category,
                                'risk_level': 'high' if category in ['configuration', 'backup', 'log_files'] else 'medium',
                                'detected_via': 'content_analysis'
                            })
                            logger.info(f"Found sensitive file: {url} [{category}]")
                    else:
                        # 对于其他类型的文件，基于状态码和响应头进行判断
                        # 检查是否为目录列表
                        if file_path.endswith('/') and \
                           any(kw in content_type for kw in ['html', 'directory listing', 'apache', 'nginx', 'iis']):
                            # 尝试获取少量内容确认是否为目录列表
                            dir_response = make_request(url, method='GET', headers=headers, timeout=timeout, 
                                                       allow_redirects=True, stream=True)
                            
                            if not dir_response:
                                dir_response = requests.get(url, headers=headers, timeout=timeout, 
                                                           allow_redirects=True, stream=True)
                            
                            dir_content = dir_response.raw.read(5000).decode('utf-8', errors='ignore')
                            
                            directory_patterns = [
                                r'<title>Index of /',
                                r'<h1>Directory Listing',
                                r'<table>.*?<tr>.*?<td><a',
                                r'<pre>.*?\\.\\./',
                                r'Parent Directory',
                                r'<a href="\\?C=N;O=D">'
                            ]
                            
                            for pattern in directory_patterns:
                                if re.search(pattern, dir_content, re.IGNORECASE):
                                    found_files.append({
                                        'url': url,
                                        'status_code': dir_response.status_code,
                                        'content_type': content_type,
                                        'category': 'directory_listing',
                                        'risk_level': 'medium',
                                        'detected_via': 'directory_structure'
                                    })
                                    logger.info(f"Found directory listing: {url}")
                                    break
                        else:
                            # 对于常见文件，使用状态码和内容类型判断
                            found_files.append({
                                'url': url,
                                'status_code': response.status_code,
                                'content_type': content_type,
                                'content_length': content_length,
                                'category': category,
                                'risk_level': 'low' if category in ['common', 'development'] else 'medium',
                                'detected_via': 'http_headers'
                            })
                            logger.info(f"Found file: {url} [{category}]")
                    
                    success = True
                    
                # 处理重定向情况
                elif response.status_code in [301, 302, 303, 307, 308]:
                    location = response.headers.get('Location', '')
                    if location and (location.startswith('/') or location.startswith(base_url)):
                        # 计算重定向后的URL
                        if location.startswith('/'):
                            redirected_url = base_url.rstrip('/') + location
                        else:
                            redirected_url = location
                        
                        # 检查是否重定向到敏感位置
                        sensitive_redirects = ['admin', 'login', 'private', 'secure', 'restricted']
                        if any(sensitive in redirected_url.lower() for sensitive in sensitive_redirects):
                            found_files.append({
                                'url': url,
                                'status_code': response.status_code,
                                'redirects_to': redirected_url,
                                'category': category,
                                'risk_level': 'low',
                                'detected_via': 'redirect_analysis'
                            })
                            logger.info(f"Found redirect to sensitive location: {url} -> {redirected_url}")
                    
                    success = True
                    
                # 403禁止访问也可能表示文件存在
                elif response.status_code == 403:
                    found_files.append({
                        'url': url,
                        'status_code': response.status_code,
                        'content_type': content_type,
                        'category': category,
                        'risk_level': 'medium',
                        'detected_via': 'access_restricted'
                    })
                    logger.info(f"Found restricted file: {url} [403 Forbidden]")
                    success = True
                    
                else:
                    success = True  # 不需要重试非成功状态码
                    
            except Exception as e:
                retry_count += 1
                # 指数退避重试
                if retry_count <= max_retries:
                    time.sleep(0.5 * (2 ** (retry_count - 1)))
                else:
                    # 记录失败但不添加到结果中
                    logger.debug(f"Failed to check {url}: {str(e)}")
    
    # 去重和排序（基于风险级别和URL）
    # 创建一个字典用于去重
    unique_files = {}
    for file_info in found_files:
        url = file_info['url']
        if url not in unique_files or \
           risk_level_to_score(file_info.get('risk_level', 'low')) > risk_level_to_score(unique_files[url].get('risk_level', 'low')):
            unique_files[url] = file_info
    
    # 转换回列表并排序
    sorted_files = list(unique_files.values())
    sorted_files.sort(key=lambda x: (risk_level_to_score(x.get('risk_level', 'low')), x['url']), reverse=True)
    
    return sorted_files

def risk_level_to_score(risk_level):
    """
    将风险级别转换为分数以便排序
    """
    risk_scores = {'high': 3, 'medium': 2, 'low': 1}
    return risk_scores.get(risk_level.lower(), 0)

def generate_vulnerability_id(vuln_type):
    """
    生成唯一的漏洞ID
    :param vuln_type: 漏洞类型
    :return: 漏洞ID字符串
    """
    timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
    random_str = hashlib.md5(f"{vuln_type}{timestamp}".encode()).hexdigest()[:8]
    return f"VULN-{vuln_type[:3].upper()}-{timestamp}-{random_str}"

def extract_form_data(html):
    """
    从HTML中提取表单数据
    :param html: HTML内容
    :return: 表单信息列表
    """
    import re
    
    forms = []
    # 简单的表单提取正则
    form_matches = re.finditer(r'<form[^>]*>(.*?)</form>', html, re.DOTALL | re.IGNORECASE)
    
    for match in form_matches:
        form_content = match.group(1)
        action_match = re.search(r'action=["\'](.*?)["\']', match.group(0), re.IGNORECASE)
        method_match = re.search(r'method=["\'](.*?)["\']', match.group(0), re.IGNORECASE)
        
        form_data = {
            'action': action_match.group(1) if action_match else '',
            'method': method_match.group(1).upper() if method_match else 'GET',
            'inputs': []
        }
        
        # 提取输入字段
        input_matches = re.finditer(r'<input[^>]*>', form_content, re.IGNORECASE)
        for input_match in input_matches:
            input_str = input_match.group(0)
            name_match = re.search(r'name=["\'](.*?)["\']', input_str, re.IGNORECASE)
            type_match = re.search(r'type=["\'](.*?)["\']', input_str, re.IGNORECASE)
            
            if name_match:
                input_data = {
                    'name': name_match.group(1),
                    'type': type_match.group(1).lower() if type_match else 'text'
                }
                form_data['inputs'].append(input_data)
        
        forms.append(form_data)
    
    return forms

def analyze_response_headers(headers):
    """
    分析HTTP响应头中的安全相关设置 - 增强版
    :param headers: HTTP响应头字典或对象
    :return: 安全问题列表（按风险级别排序）
    """
    import re
    from app.utils.logger import logger
    
    security_issues = []
    
    # 标准化headers为字典格式
    headers_dict = {}
    if isinstance(headers, dict):
        headers_dict = headers
    elif hasattr(headers, 'items'):
        # 处理requests.Response对象的headers
        headers_dict = dict(headers.items())
    
    # 转换所有头名称为小写以便大小写不敏感的比较
    headers_lower = {k.lower(): v for k, v in headers_dict.items()}
    
    # 定义关键安全头及其建议值/配置
    security_headers = {
        'x-frame-options': {
            'required': True,
            'valid_values': ['DENY', 'SAMEORIGIN', 'ALLOW-FROM'],
            'description': '防止点击劫持攻击',
            'risk_level': 'high',
            'recommendation': '设置为DENY或SAMEORIGIN以防止网站被嵌入到iframe中'
        },
        'content-security-policy': {
            'required': True,
            'description': '限制资源加载，防止XSS和数据注入攻击',
            'risk_level': 'high',
            'recommendation': '实施严格的CSP策略，避免使用unsafe-inline和unsafe-eval',
            'check_function': lambda v: not ("'unsafe-inline'" in v and "'unsafe-eval'" in v)
        },
        'x-xss-protection': {
            'required': True,
            'valid_values': ['1; mode=block'],
            'description': '启用浏览器XSS过滤机制',
            'risk_level': 'medium',
            'recommendation': '设置为"1; mode=block"以启用严格的XSS过滤'
        },
        'x-content-type-options': {
            'required': True,
            'valid_values': ['nosniff'],
            'description': '防止MIME类型嗅探',
            'risk_level': 'medium',
            'recommendation': '设置为"nosniff"以防止浏览器猜测内容类型'
        },
        'strict-transport-security': {
            'required': True,
            'description': '强制使用HTTPS连接',
            'risk_level': 'high',
            'recommendation': '设置合理的max-age值（如一年）并考虑包含includeSubDomains和preload',
            'check_function': lambda v: 'max-age=' in v and int(re.search(r'max-age=(\d+)', v).group(1)) >= 31536000
        },
        'referrer-policy': {
            'required': True,
            'valid_values': ['no-referrer', 'no-referrer-when-downgrade', 'strict-origin-when-cross-origin', 
                            'origin', 'origin-when-cross-origin', 'same-origin', 'strict-origin'],
            'description': '控制HTTP Referer头的发送',
            'risk_level': 'medium',
            'recommendation': '根据需要选择合适的策略，通常推荐使用"strict-origin-when-cross-origin"或"strict-origin"'
        },
        'permissions-policy': {
            'required': False,
            'description': '限制网站可以使用的浏览器功能',
            'risk_level': 'medium',
            'recommendation': '明确指定允许使用的功能，禁用不需要的功能'
        },
        'x-permitted-cross-domain-policies': {
            'required': False,
            'valid_values': ['none', 'master-only', 'by-content-type', 'by-ftp-filename', 'all'],
            'description': '控制Flash和PDF文件如何处理跨域请求',
            'risk_level': 'low',
            'recommendation': '设置为"none"除非有特定需要'
        },
        'feature-policy': {
            'required': False,
            'description': '旧版Permissions-Policy，限制网站可以使用的浏览器功能',
            'risk_level': 'medium',
            'recommendation': '考虑使用新版Permissions-Policy，但如果使用此头，应明确限制功能'
        },
        'access-control-allow-origin': {
            'required': False,
            'description': '控制跨域资源共享(CORS)',
            'risk_level': 'high',
            'recommendation': '避免使用通配符(*)，仅允许特定域名访问',
            'check_function': lambda v: v != '*' or '*' in v and 'credentials' in headers_lower and headers_lower['credentials'] != 'include'
        },
        'server': {
            'required': False,
            'description': '服务器标识',
            'risk_level': 'low',
            'recommendation': '移除或修改以隐藏服务器版本信息，减少信息泄露'
        },
        'x-powered-by': {
            'required': False,
            'description': '显示使用的框架或技术',
            'risk_level': 'low',
            'recommendation': '移除以减少信息泄露'
        },
        'pragma': {
            'required': False,
            'description': 'HTTP/1.0缓存控制',
            'risk_level': 'low',
            'recommendation': '确保包含"no-cache"以防止敏感数据缓存'
        },
        'cache-control': {
            'required': True,
            'description': '控制缓存行为',
            'risk_level': 'medium',
            'recommendation': '对于敏感页面，设置为"no-store, no-cache, must-revalidate"'
        }
    }
    
    # 检查每个安全头
    for header_name, header_info in security_headers.items():
        if header_name in headers_lower:
            # 头存在，检查其值是否符合安全标准
            header_value = headers_lower[header_name]
            
            # 如果有特定的检查函数，使用它
            if 'check_function' in header_info:
                if not header_info['check_function'](header_value):
                    security_issues.append({
                        'type': 'header_value_weak',
                        'header': header_name,
                        'current_value': header_value,
                        'description': f'头 {header_name} 的配置不够安全',
                        'risk_level': header_info['risk_level'],
                        'recommendation': header_info['recommendation']
                    })
                    logger.debug(f"Weak {header_name} header detected: {header_value}")
            
            # 检查是否在有效值列表中
            elif 'valid_values' in header_info:
                # 对于允许部分匹配的值（如X-Frame-Options的ALLOW-FROM）
                if header_info['valid_values'][0].startswith('ALLOW-'):
                    valid = False
                    for valid_val in header_info['valid_values']:
                        if valid_val in header_value or header_value.startswith(valid_val):
                            valid = True
                            break
                    
                    if not valid:
                        security_issues.append({
                            'type': 'header_value_invalid',
                            'header': header_name,
                            'current_value': header_value,
                            'description': f'头 {header_name} 的值无效或不安全',
                            'risk_level': header_info['risk_level'],
                            'recommendation': header_info['recommendation']
                        })
                        logger.debug(f"Invalid {header_name} header value: {header_value}")
                else:
                    # 严格匹配的值
                    if header_value not in header_info['valid_values']:
                        security_issues.append({
                            'type': 'header_value_invalid',
                            'header': header_name,
                            'current_value': header_value,
                            'description': '头 {} 的值应为: {}'.format(header_name, ', '.join(header_info['valid_values'])),
                            'risk_level': header_info['risk_level'],
                            'recommendation': header_info['recommendation']
                        })
                        logger.debug(f"Invalid {header_name} header value: {header_value}")
        else:
            # 头不存在
            if header_info['required']:
                security_issues.append({
                    'type': 'missing_header',
                    'header': header_name,
                    'description': f"缺少安全头 {header_name}，{header_info['description']}",
                    'risk_level': header_info['risk_level'],
                    'recommendation': header_info['recommendation']
                })
                logger.debug(f"Missing required security header: {header_name}")
    
    # 检查Cookie安全标志
    if 'set-cookie' in headers_lower:
        # 处理多个Cookie（可能是列表或单个字符串）
        cookies = []
        cookie_header = headers_lower['set-cookie']
        
        # 检查是否为列表或包含多个cookie
        if isinstance(cookie_header, list):
            cookies = cookie_header
        elif ';' in cookie_header:
            # 简单分割，实际应更复杂但基本够用
            cookies_parts = cookie_header.split(';')
            # 基本处理：每个cookie应该有name=value格式
            current_cookie = ''
            for part in cookies_parts:
                part = part.strip()
                if '=' in part and not part.startswith(('HttpOnly', 'Secure', 'SameSite', 'Path=', 'Domain=', 'Max-Age=', 'Expires=')):
                    # 这可能是一个新cookie的开始
                    if current_cookie:
                        cookies.append(current_cookie)
                    current_cookie = part
                else:
                    current_cookie += ';' + part
            if current_cookie:
                cookies.append(current_cookie)
        else:
            cookies = [cookie_header]
        
        for i, cookie in enumerate(cookies):
            cookie_parts = cookie.split(';')
            cookie_name = cookie_parts[0].split('=')[0].strip()
            cookie_lower = cookie.lower()
            
            # 检查HttpOnly标志
            if 'httponly' not in cookie_lower:
                security_issues.append({
                    'type': 'cookie_missing_flag',
                    'cookie': cookie_name,
                    'flag': 'HttpOnly',
                    'description': f'Cookie "{cookie_name}" 缺少HttpOnly标志，可能容易受到XSS攻击导致cookie被窃取',
                    'risk_level': 'high',
                    'recommendation': '为所有Cookie添加HttpOnly标志，防止JavaScript访问'
                })
                logger.debug(f"Cookie missing HttpOnly flag: {cookie_name}")
            
            # 检查Secure标志
            if 'secure' not in cookie_lower:
                security_issues.append({
                    'type': 'cookie_missing_flag',
                    'cookie': cookie_name,
                    'flag': 'Secure',
                    'description': f'Cookie "{cookie_name}" 缺少Secure标志，可能在非HTTPS连接上传输',
                    'risk_level': 'high',
                    'recommendation': '为所有Cookie添加Secure标志，确保仅通过HTTPS传输'
                })
                logger.debug(f"Cookie missing Secure flag: {cookie_name}")
            
            # 检查SameSite标志
            if 'samesite' not in cookie_lower:
                security_issues.append({
                    'type': 'cookie_missing_flag',
                    'cookie': cookie_name,
                    'flag': 'SameSite',
                    'description': f'Cookie "{cookie_name}" 缺少SameSite标志，可能容易受到CSRF攻击',
                    'risk_level': 'medium',
                    'recommendation': '为所有Cookie添加SameSite=Lax或SameSite=Strict标志'
                })
                logger.debug(f"Cookie missing SameSite flag: {cookie_name}")
            else:
                # 检查SameSite值是否为推荐值
                same_site_match = re.search(r'samesite\s*=\s*(\w+)', cookie_lower)
                if same_site_match:
                    same_site_value = same_site_match.group(1)
                    if same_site_value not in ['strict', 'lax']:
                        security_issues.append({
                            'type': 'cookie_flag_value_weak',
                            'cookie': cookie_name,
                            'flag': 'SameSite',
                            'current_value': same_site_value,
                            'description': f'Cookie "{cookie_name}" 的SameSite值不是推荐值',
                            'risk_level': 'low',
                            'recommendation': '使用SameSite=Lax（默认）或SameSite=Strict，避免使用SameSite=None除非必要'
                        })
                        logger.debug(f"Cookie has weak SameSite value: {cookie_name} = {same_site_value}")
            
            # 检查是否有敏感cookie名称
            sensitive_cookie_names = ['session', 'token', 'auth', 'login', 'password', 'pass', 'credential', 
                                     'secret', 'api', 'key', 'jwt', 'oauth']
            if any(sensitive in cookie_name.lower() for sensitive in sensitive_cookie_names):
                # 敏感cookie需要特别关注其安全标志
                if 'httponly' not in cookie_lower or 'secure' not in cookie_lower:
                    security_issues.append({
                        'type': 'sensitive_cookie_insecure',
                        'cookie': cookie_name,
                        'description': f'敏感Cookie "{cookie_name}" 缺少关键安全标志',
                        'risk_level': 'critical',
                        'recommendation': '敏感Cookie必须同时设置HttpOnly和Secure标志'
                    })
                    logger.debug(f"Sensitive cookie missing security flags: {cookie_name}")
    
    # 检查缓存相关头的组合
    if ('cache-control' in headers_lower and 'no-store' not in headers_lower['cache-control'].lower() and 
        'pragma' not in headers_lower):
        security_issues.append({
            'type': 'cache_control_insecure',
            'description': '缓存控制配置不够严格，可能导致敏感信息被缓存',
            'risk_level': 'medium',
            'recommendation': '对于包含敏感信息的页面，设置Cache-Control: no-store, no-cache, must-revalidate和Pragma: no-cache'
        })
        logger.debug("Insecure cache control detected")
    
    # 检查内容类型是否指定了字符集
    if 'content-type' in headers_lower:
        content_type = headers_lower['content-type']
        if 'charset=' not in content_type.lower() and any(media_type in content_type.lower() for media_type in ['text/html', 'text/plain', 'application/json']):
            security_issues.append({
                'type': 'content_type_missing_charset',
                'description': 'Content-Type头缺少字符集指定，可能导致字符编码问题和XSS漏洞',
                'risk_level': 'low',
                'recommendation': '始终在Content-Type头中明确指定字符集，如charset=utf-8'
            })
            logger.debug("Content-Type missing charset specification")
    
    # 检查CORS配置
    if 'access-control-allow-origin' in headers_lower and headers_lower['access-control-allow-origin'] == '*':
        if 'access-control-allow-credentials' in headers_lower and headers_lower['access-control-allow-credentials'].lower() == 'true':
            security_issues.append({
                'type': 'cors_misconfiguration',
                'description': 'CORS配置存在安全风险：允许所有来源(*)同时允许凭证(credentials)',
                'risk_level': 'high',
                'recommendation': '不要同时设置Access-Control-Allow-Origin为*和Access-Control-Allow-Credentials为true，应明确指定允许的来源域名'
            })
            logger.debug("Insecure CORS configuration detected: * with credentials")
    
    # 检查是否启用了不安全的方法
    if 'access-control-allow-methods' in headers_lower:
        methods = headers_lower['access-control-allow-methods'].lower()
        risky_methods = ['delete', 'put', 'connect', 'trace']
        for method in risky_methods:
            if method in methods and not any(auth_header in headers_lower for auth_header in ['authorization', 'www-authenticate']):
                security_issues.append({
                    'type': 'risky_http_methods_enabled',
                    'method': method.upper(),
                    'description': f'启用了高风险HTTP方法 {method.upper()} 但未检测到身份验证机制',
                    'risk_level': 'medium',
                    'recommendation': '限制允许的HTTP方法，或确保对高风险方法实施适当的身份验证'
                })
                logger.debug(f"Risky HTTP method enabled without authentication: {method.upper()}")
    
    # 计算安全分数（满分100）
    total_possible_issues = sum(1 for h in security_headers.values() if h['required']) + 3  # 基本安全头 + Cookie三个标志
    detected_issues = sum(1 for issue in security_issues if issue['risk_level'] in ['high', 'critical'])
    
    # 计算安全分数（简化版）
    security_score = max(0, 100 - (detected_issues * 10))
    
    # 添加安全分数到结果中
    result = {
        'security_score': security_score,
        'security_level': get_security_level(security_score),
        'issues': sort_issues_by_risk(security_issues)
    }
    
    logger.info(f"HTTP security headers analysis completed. Score: {security_score}/100, Issues: {len(security_issues)}")
    return result

def get_security_level(score):
    """
    根据安全分数返回安全级别
    """
    if score >= 90:
        return 'A+'
    elif score >= 80:
        return 'A'
    elif score >= 70:
        return 'B'
    elif score >= 60:
        return 'C'
    elif score >= 50:
        return 'D'
    else:
        return 'F'

def sort_issues_by_risk(issues):
    """
    根据风险级别对问题进行排序
    """
    risk_order = {'critical': 0, 'high': 1, 'medium': 2, 'low': 3}
    
    return sorted(issues, key=lambda x: risk_order.get(x.get('risk_level', 'low'), 3))